package com.spider.moudle.crawler.forum;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONException;
import com.alibaba.fastjson.JSONObject;
import com.spider.moudle.api.Downloader;
import com.spider.moudle.api.realize.AbstractCrawler;
import com.spider.moudle.api.realize.AbstractReplyCrawler;
import com.spider.moudle.constant.Constant;
import com.spider.moudle.entity.Area;
import com.spider.moudle.entity.ResponseEntity;
import com.spider.moudle.entity.Setting;
import com.spider.moudle.entity.SpeechBean;
import com.spider.moudle.entity.Task;
import com.spider.moudle.entity.TaskType;
import com.spider.moudle.entity.UserBean;
import com.spider.moudle.entity.WebType;
import com.spider.moudle.factory.DownloaderFactory;
import com.spider.moudle.factory.ExecutorServiceFactoy;
import com.spider.moudle.factory.NamedThreadFactory;
import com.spider.moudle.task.TaskManager;
import com.spider.moudle.util.CommonUtil;
import com.spider.moudle.util.DateUtil;
import com.spider.moudle.util.SettingsUtil;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.FutureTask;

/**
 * Created by Linbo Hu on 2017/11/21.
 * 天涯社区
 */
public class TianYaCrawler extends AbstractCrawler {

    /**
     * 这个是每个网站的个性化配置，是从settings.json里面读取到的(必须)
     */
    private Setting setting = SettingsUtil.getSettingBySite("TianYa");

    /**
     * 用户主页地址(非必须)
     */
    private String index = setting.getIndex();

    /**
     * 生成任务 (这个方法是根据不同的网址个性化定制的，也是开发人员重点要编写的内容，这里是针对天涯论坛的生成任务代码)
     */
    @Override
    public void generateTask() {
        ExecutorService executorService = null;
        try {
            ResponseEntity entity = null;
            try {
                entity = downloader.get(index, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                Document document = entity.getDocument();
                Elements elements = document.select("div[class=\"nav_child_box\"] ul li a");
                if (elements.isEmpty()) {
                    return;
                }
                //这里通过设置线程池的最大值来控制生成任务的线程数
                executorService = ExecutorServiceFactoy.buildExecutorService(setting.getTask_thread_count(), ExecutorServiceFactoy.getName(setting.getName(), setting.getIndex()), new NamedThreadFactory(setting.getName(), "生成任务"));
                for (int i = 0; i < elements.size(); i++) {
                    Element element = elements.get(i);
                    String url = element.attr("href");
                    if (!url.contains("http")) {
                        url = index + url;
                    }
                    String text = element.text();
                    Area area = Area.search(text);
                    if (null == area) {
                        if (url.contains("-")) {
                            TianYaTaskCrawler default_task = new TianYaTaskCrawler(area, url);
                            executorService.submit(default_task);
                            String newestUrl = url.substring(url.lastIndexOf("/") + 1);
                            newestUrl = newestUrl.substring(newestUrl.indexOf("-") + 1, newestUrl.lastIndexOf("-"));
                            newestUrl = "http://bbs.tianya.cn/list.jsp?item=" + newestUrl + "&order=1";
                            TianYaTaskCrawler new_task = new TianYaTaskCrawler(area, newestUrl);
                            executorService.submit(new_task);
                        }
                    } else {
                        //如果是地域
                        ResponseEntity regionEntity = downloader.get(url, true);
                        if (entity.getStatus_code() == 200) {
                            Document regionDocument = regionEntity.getDocument();
                            Elements regionElements = regionDocument.select("div[class=\"nav_child_box\"] ul li a");
                            if (regionElements.isEmpty()) {
                                return;
                            }
                            for (Element regionElement : regionElements) {
                                String regionUrl = regionElement.attr("href");
                                if (!regionUrl.contains("http")) {
                                    regionUrl = index + regionUrl;
                                }
                                String regionText = element.text();
                                Area regionArea = Area.search(regionText);
                                TianYaTaskCrawler tianYaTaskCrawler = new TianYaTaskCrawler(regionArea, regionUrl);
                                executorService.submit(tianYaTaskCrawler);
                            }

                        }
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (null != executorService) {
                executorService.shutdown();
                while (!executorService.isTerminated()) {
                    try {
                        Thread.sleep(1500);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
    }

    /**
     * 爬取单条言论 (这个方法是根据不同的网址个性化定制的，也是开发人员重点要编写的内容，这里是针对天涯论坛的爬虫代码)
     *
     * @param task
     * @return
     * @throws Exception
     */
    @Override
    public JSONObject crawlItem(Task task) throws Exception {
        //抓言论的爬虫代码块
        if (task.getTaskType() == TaskType.SPEECH) {
            SpeechBean speechBean = new SpeechBean();
            JSONObject item = new JSONObject();
            String url = task.getUrl();
            speechBean.setUrl(url);
            speechBean.setArea(task.getArea());
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            int page = 1;
            if (entity != null && entity.getStatus_code() == 200) {
                try {
                    Document document = entity.getDocument();
                    Elements pageInfos = document.select("#bd");
                    if (null != pageInfos && pageInfos.size() > 0) {
                        Element pageInfo = pageInfos.get(0);
                        String content = "";
                        if (null != pageInfo.select(".bbs-content") && pageInfo.select(".bbs-content").size() > 0 &&
                                !"".equals(pageInfo.select(".bbs-content").get(0).html())) {
                            String title = document.select("#post_head").select(".s_title").select("span").get(0)
                                    .text();
                            String read_String = pageInfo.select("#post_head").select(".atl-info").select("span").get
                                    (2).text();
                            String read_Str = read_String.split("：")[1].trim();
                            // 阅读量
                            int read_count = Integer.parseInt(read_Str);
                            String reply_String = pageInfo.select("#post_head").select(".atl-info").select("span")
                                    .get(3).text();
                            String reply_Str = reply_String.split("：")[1].trim();
                            // 回复量
                            int reply_count = Integer.parseInt(reply_Str);
                            Elements pageI = document.select("#post_head").select("form").select("a");
                            if (null != pageI && pageI.size() > 0) {
                                String pageString = pageI.get(pageI.size() - 2).text();
                                page = Integer.valueOf(pageString);
                            }
                            // 发布时间
                            String publish_time = "";
                            String publish_time_text = pageInfo.select("#post_head").select(".atl-info").select
                                    ("span").get(1).text();
                            String reltime = publish_time_text.substring(3, publish_time_text.length());
                            publish_time = getTime(reltime);
                            if(CommonUtil.isNull(publish_time)) {
                                return null;
                            }
                            // 发布内容
                            content = pageInfo.select(".bbs-content").get(0).text();
                            List<String> img_urls = new ArrayList<>();
                            Elements imgElements = pageInfo.select(".bbs-content").get(0).select("img[original]");
                            for (int imgIndex = 0; imgIndex < imgElements.size(); imgIndex++) {
                                Element imgEle = imgElements.get(imgIndex);
                                String img_url = imgEle.absUrl("origina" + "l");
                                img_urls.add(img_url);
                            }
                            // 发布者的ID
                            String publish_user_id = pageInfo.select("#post_head").select(".atl-info").select("span").get(0).select("a").get(0).attr("uid").toString();
                            // 发布用户
                            String publish_user = pageInfo.select("#post_head").select(".atl-info").select("span")
                                    .get(0).select("a").get(0).text();
                            String publish_user_photo = "http://tx.tianyaui.com/logo/" + publish_user_id;
                            String[] urlParam = url.split("-");
                            // 帖子的Id
                            String id = urlParam[2];

                            speechBean.setReply_count(reply_count);
                            speechBean.setRead_count(read_count);
                            speechBean.setTitle(CommonUtil.handle(title));
                            speechBean.setUrl(url);
                            speechBean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                    publish_time));

                            Date now = new Date();

                            speechBean.setSpider_time(now);
                            speechBean.setReproduce_count(0);
                            speechBean.setContent(CommonUtil.handle(content));
                            speechBean.setPublish_user_id(publish_user_id);
                            //将发帖人添加到任务
                            if (CommonUtil.isNotNull(publish_user_id)) {
                                String userKey = setting.getName() + "_" + publish_user_id;
                                String userUrl = "http://www.tianya.cn/" + publish_user_id;
                                Task hometask = new Task(userUrl, TaskType.USER, WebType.FORLUM, userKey);
                                hometask.put("publish_user", publish_user_id);
                                hometask.put("publish_user_photo", publish_user_id);
                                TaskManager.add(setting.getUser_task(), hometask);
                            }

                            speechBean.setPublish_user_jsonid(setting.getName() + "_" + publish_user_id);
                            speechBean.setPublish_user(publish_user);
                            speechBean.setPublish_user_photo(publish_user_photo);
                            speechBean.setId(id);
                            speechBean.setImg_urls(img_urls);
                            Elements replyEntities = pageInfo.select("div[class=\"atl-main\"]").select("div[class=\"atl-item\"]");
                            List<SpeechBean> reply_nodes = new ArrayList<>();
                            Elements video_elements = document.select("embed[class=\"video\"]");
                            List<String> video_list = new ArrayList<>();
                            if(!video_elements.isEmpty()) {
                                Element element = video_elements.get(0);
                                String video_url = element.attr("src");
                                video_list.add(video_url);
                            }
                            video_elements = document.select("div[class=\"bbs-content clearfix\"]");
                            if(!video_elements.isEmpty()) {
                                Element element = video_elements.get(0).selectFirst("span");
                                if(element != null) {
                                    String video_url = element.attr("videoId");
                                    if(CommonUtil.isNotNull(video_url)) {
                                        video_url = "http://api.video.tianya.cn/v/play/" + video_url;
                                        video_list.add(video_url);
                                    }
                                }
                            }
                            speechBean.setVideo_urls(video_list);
                            // 爬取第一页的回复
                            for (int i = 0; i < replyEntities.size(); i++) {
                                Element reply_element = replyEntities.get(i);
                                Elements replyUserInfo = reply_element.select(".atl-head").select(".atl-info").select("span");
                                SpeechBean reply = new SpeechBean();
                                String replyUser = "";
                                Elements es = replyUserInfo.select(".authi").select("a");
                                if (null != replyUserInfo && null != replyUserInfo.get(0).select("a") &&
                                        replyUserInfo.get(0).select("a").size() > 0) {
                                    replyUser = replyUserInfo.get(0).select("a").get(0).text();
                                }
                                String replyUserId = replyUserInfo.get(0).select("a").get(0).attr("uid").toString();
                                String replyUserPhoto = "http://tx.tianyaui.com/logo/" + replyUserId;
                                reply_element.select(".atl-content").get(0).select(".atl-reply").remove();
                                String replyContent = reply_element.select(".atl-content div[class=\"bbs-content\"]").text();
                                List<String> reply_img_urls = new ArrayList<>();
                                Elements replyImgElemnts = reply_element.select(".atl-content").select("img[original]");

                                for (int imgIndex = 0; imgIndex < replyImgElemnts.size(); imgIndex++) {
                                    Element imgEle = replyImgElemnts.get(imgIndex);
                                    String img_url = imgEle.attr("original");
                                    reply_img_urls.add(img_url);
                                }
                                String replyTimeStr = replyUserInfo.get(1).text();
                                String replyTimeStr2 = replyTimeStr.substring(3, replyTimeStr.length());
                                String replyTime = getTime(replyTimeStr2);
                                String replyUrl = url + "#" + reply_element.id();
                                String replyId = reply_element.attr("replyid");
                                reply.setId(replyId);
                                reply.setPublish_user(replyUser);
                                reply.setPublish_user_id(replyUserId);
                                reply.setPublish_user_jsonid(setting.getName() + "_" + replyUserId);
                                reply.setPublish_user_photo(replyUserPhoto);
                                if (CommonUtil.isNotNull(replyUserId)) {
                                    if (replyUserId.contains("http")) {
                                        replyUserId = replyUserId.substring(replyUserId.lastIndexOf("/"));
                                    } else {

                                    }
                                    if (replyUserId.startsWith("/")) {
                                        replyUserId = replyUserId.substring(1);
                                    }
                                    String userKey = setting.getName() + "_" + replyUserId;
                                    String userUrl = "http://www.tianya.cn/" + replyUserId;
                                    Task hometask = new Task(userUrl, TaskType.USER, WebType.FORLUM, userKey);
                                    hometask.put("publish_user", replyUser);
                                    hometask.put("publish_user_photo", replyUserPhoto);
                                    TaskManager.add(setting.getUser_task(), hometask);
                                }
                                reply.setContent(replyContent.trim());
                                reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, replyTime));
                                reply.setUrl(replyUrl);
                                reply.setParent_id(id);
                                reply.setAncestor_id(id);
                                reply.setImg_urls(reply_img_urls);
                                reply.setArea(task.getArea());
                                //获取回复的回复
                                Elements reply_reply_elements = reply_element.select("div[class=\"item-reply-view\"] ul li");
                                List<SpeechBean> reply_reply_nodes = new ArrayList<>();
                                for(Element element : reply_reply_elements) {
                                    SpeechBean reply_reply_bean = new SpeechBean();
                                    String reply_reply_publish_user_id = element.attr("_userid");
                                    reply_reply_bean.setPublish_user_id(reply_reply_publish_user_id);
                                    String reply_reply_publish_time = element.attr("_replytime");
                                    reply_reply_bean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_reply_publish_time));
                                    Elements p_elements = element.select("p");
                                    if(p_elements.isEmpty()) {
                                        continue;
                                    }
                                    String reply_reply_publish_user = element.attr("_username");
                                    String reply_reply_publish_user_photo = "http://tx.tianyaui.com/logo/" + reply_reply_publish_user_id;
                                    reply_reply_bean.setPublish_user_photo(reply_reply_publish_user_photo);
                                    reply_reply_bean.setPublish_user(reply_reply_publish_user);
                                    Element content_elment = element.select("span[class=\"ir-content\"]").get(0);
                                    String reply_reply_content = content_elment.text();
                                    reply_reply_bean.setContent(reply_reply_content);
                                    reply_reply_bean.setPublish_user_jsonid(setting.getName() + "_" + reply_reply_publish_user_id);
                                    reply_reply_bean.setParent_id(replyId);
                                    reply_reply_bean.setAncestor_id(id);
                                    reply_reply_bean.setUrl(replyUrl);
                                    reply_reply_nodes.add(reply_reply_bean);
                                }
                                reply.setReply_count(reply_reply_nodes.size());
                                //添加回复的回复
                                reply.setReply_nodes(reply_reply_nodes);
                                reply_nodes.add(reply);
                            }
                            //爬虫最大页码的限制，超过最大页的帖子就只抓到最大页码数
                            if (page >= setting.getMax_page_count()) {
                                page = setting.getMax_page_count();
                            }
                            // 第二页以及以后的页码是通过多线程去抓取的
                            if (page >= 2) {
                                int threadNum = setting.getReply_thread_count();
                                List<FutureTask<List<SpeechBean>>> futureTasks = new ArrayList<>();
                                ArrayList<ArrayList<Integer>> splitPages = CommonUtil.splitPages(2, page, threadNum);
                                ExecutorService executorService = null;
                                try {
                                    executorService = ExecutorServiceFactoy.buildExecutorService(threadNum, ExecutorServiceFactoy.getName(setting.getName(), url), new NamedThreadFactory(setting.getName(), "回复"));
                                    for (int i = 0; i < splitPages.size(); i++) {
                                        ReplyCrawler callable = new ReplyCrawler(splitPages.get(i), url, id, id, task.getArea());
                                        FutureTask<List<SpeechBean>> futureTask = new FutureTask<>(callable);
                                        futureTasks.add(futureTask);
                                        executorService.submit(futureTask);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                    log.error("天涯社区获取回复线程执行异常");
                                } finally {
                                    if (null != executorService) {
                                        executorService.shutdown();
                                    }
                                }

                                for (FutureTask<List<SpeechBean>> futureTask : futureTasks) {
                                    reply_nodes.addAll(futureTask.get());
                                }
                                Collections.sort(reply_nodes, (o1, o2) -> {
                                    Date s1 = o1.getPublish_time();
                                    Date s2 = o2.getPublish_time();
                                    int flag = s1.compareTo(s2);
                                    return flag;
                                });
                            }
                            speechBean.setReply_nodes(reply_nodes);
                        }
                    }
                    if(CommonUtil.isNull(speechBean.getPublish_time())) {
                        return null;
                    }
                    item.put("data", speechBean.toJSONObject());
                    return item;
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error(setting.getName() + "获取言论数据解析错误: " + url);
                }
            }
            //抓用户数据的爬虫代码
        } else if (task.getTaskType() == TaskType.USER) {
            String userKey = task.getTaskKey();
            if (CommonUtil.isNull(userKey)) {
                System.err.println("User key is not empty where task type belong to home page");
                return null;
            }
            JSONObject item = new JSONObject();
            UserBean data;
            String url = task.getUrl();
            ResponseEntity entity = null;
            try {
                entity = downloader.get(url, true);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
            if (entity != null && entity.getStatus_code() == 200) {
                Date register_time = null;
                String gender = null;
                try {
                    Document document = entity.getDocument();
                    Elements script_elementss = document.select("script");
                    if(script_elementss.size() < 3) {
                        return null;
                    }
                    Element script_elements = document.select("script").get(2);
                    String userinfo = script_elements.toString();
                    String publish_user = (String) task.get("publish_user");
                    String publish_user_id = userKey.substring(userKey.indexOf("_") + 1);
                    try {
                        if(userinfo.contains("{") && userinfo.contains("}")) {
                            userinfo = userinfo.substring(userinfo.indexOf("{"), userinfo.indexOf("}")+1).replace("\r", "").replace("\n", "").replace("//他，她，我", "").trim();
                            JSONObject user_info_object = JSON.parseObject(userinfo);
                            gender = user_info_object.getString("gender");
                            String register_time_string = user_info_object.getString("regTime");
                            //如果相同则重新在用户页面抓取
                            if(publish_user.equals(publish_user_id)) {
                                publish_user = user_info_object.getString("userName");
                                publish_user_id = user_info_object.getString("userId");
                            }
                            register_time = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, register_time_string);
                        }
                    } catch (Exception e) {
                        System.out.println(userinfo);
                    }
                    Elements relations_elements = document.select("div[class=\"relations\"]");
                    int follow_count = 0;
                    int fans_count = 0;
                    if(!relations_elements.isEmpty()) {
                        Element relations_element = relations_elements.get(0);
                        Elements relation_box = relations_element.select("div[class=\"link-box\"]");
                        String follow_count_string = relation_box.get(0).select("a").text().trim();
                        follow_count = Integer.parseInt(follow_count_string);
                        String fans_count_string = relation_box.get(1).select("a").text().trim();
                        fans_count = Integer.parseInt(fans_count_string);
                    }
                    data = new UserBean();
                    data.setRegister_time(register_time);
                    data.setSex(gender);
                    data.setJson_id(userKey);
                    data.setUrl(url);
                    data.setPublish_user_id(publish_user_id);
                    data.setPublish_user(publish_user);
                    String publish_user_photo = (String) task.get("publish_user_photo");
                    if(!publish_user_photo.contains("http")) {
                        publish_user_photo = "http://tx.tianyaui.com/logo/" + publish_user_photo;
                    }
                    data.setPublish_user_photo(publish_user_photo);
                    String article_reply_url = "http://www.tianya.cn/api/bbsuser?method=userinfo.ice" + "" + "" + "" + ""
                            + ".getUserTotalList&params.userId=" + publish_user_id;
                    int article_count = 0;
                    int reply_article_count = 0;
                    ResponseEntity article_reply_entity = null;
                    try {
                        article_reply_entity = downloader.get(article_reply_url, true);
                    } catch (Exception e) {
                        log.error(setting.getName() + "访问：" + article_reply_url + "出错", e);
                    }
                    if (article_reply_entity != null && article_reply_entity.getStatus_code() == 200) {
                        try {
                            String response_body = article_reply_entity.getResponse_body();
                            JSONObject jsonObject = JSON.parseObject(response_body);
                            article_count = jsonObject.getJSONObject("data").getInteger("compose");
                            reply_article_count = jsonObject.getJSONObject("data").getInteger("reply");
                        } catch (Exception e) {
                            log.error("天涯社区获取发帖量和回复量解析数据出错: " + url, e);
                        }
                    }
                    data.setFriend_details(new ArrayList<>());
                    data.setArticle_count(article_count);
                    data.setReply_article_count(reply_article_count);
                    data.setVisit_count(0);
                    //关注(有关注用户就需要爬关注用户信息)
                    List<UserBean> follow_details = new ArrayList<>();
                    int pageTotal = 1;
                    //这里其实只抓取了第一页的数据, pageTotal始终是1
                    for (int pageNum = 1; pageNum <= pageTotal; pageNum++) {
                        String follow_details_url = "http://www.tianya.cn/api/tw?method=following.ice.select&params" + ""
                                + ".userId=" + publish_user_id + "&params.pageSize=50&params.pageNo=" + pageNum;
                        ResponseEntity follow_details_entity = null;
                        try {
                            follow_details_entity = downloader.get(follow_details_url, true);
                        } catch (Exception e) {
                            log.error(setting.getName() + "访问：" + follow_details_url + "出错", e);
                        }
                        if (follow_details_entity == null) {
                            continue;
                        }
                        if (follow_details_entity != null && follow_details_entity.getStatus_code() == 200) {
                            try {
                                String response_body = follow_details_entity.getResponse_body();
                                JSONObject jsonObject = null;
                                try {
                                    jsonObject = JSON.parseObject(response_body);
                                } catch (Exception e) {
                                    continue;
                                }
                                if (!jsonObject.getJSONObject("data").containsKey("user")) {
                                    continue;
                                }
                                JSONArray array = jsonObject.getJSONObject("data").getJSONArray("user");
                                if (!array.isEmpty()) {
                                    for (int i = 0; i < array.size(); i++) {
                                        JSONObject object = array.getJSONObject(i);
                                        String id = object.getString("id");
                                        String photo = "http://tx.tianyaui.com/logo/small/" + id;
                                        String name = object.getString("name");
                                        String url_ = "http://www.tianya.cn/" + id;
                                        UserBean result_object = new UserBean();
                                        result_object.setUrl(url_);
                                        result_object.setPublish_user_id(id);
                                        result_object.setPublish_user_photo(photo);
                                        result_object.setPublish_user(name);
                                        result_object.setJson_id(setting.getName() + "_" + id);
                                        follow_details.add(result_object);
                                    }
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                                log.error("天涯社区获取关注数据出错: " + follow_details_url, e);
                            }
                        }
                    }


                    data.setFollow_details(follow_details);
                    data.setFollow_count(follow_count);
                    //如果关注数大于0，但是关注详情为空则不要这条数据
                    if(follow_count > 0 && follow_details.isEmpty()) {
                        return null;
                    }
                    //粉丝(有粉丝就需要爬粉丝信息)
                    List<UserBean> fans_details = new ArrayList<>();
                    pageTotal = 1;
                    //这里其实只抓取了第一页的数据, pageTotal始终是1
                    for (int pageNum = 1; pageNum <= pageTotal; pageNum++) {
                        String follow_details_url = "http://www.tianya.cn/api/tw?method=follower.ice.select&params" + ""
                                + ".userId=" + publish_user_id + "&params.pageNo=" + pageNum + "&params.pageSize=50";
                        ResponseEntity follow_details_entity = null;
                        try {
                            follow_details_entity = downloader.get(follow_details_url, true);
                        } catch (Exception e) {
                            log.error(setting.getName() + "访问：" + follow_details_url + "出错", e);
                        }
                        if (follow_details_entity == null) {
                            continue;
                        }
                        if (follow_details_entity.getStatus_code() == 200) {
                            try {
                                String response_body = follow_details_entity.getResponse_body();
                                JSONObject jsonObject = null;
                                try {
                                    jsonObject = JSON.parseObject(response_body);
                                } catch (JSONException e) {
                                    continue;
                                }
                            /*if(pageNum == 1) {
                                int dataTotal = jsonObject.getJSONObject("data").getInteger("total");
                                pageTotal = (dataTotal + 49) / 50;
                            }*/
                                if (!jsonObject.getJSONObject("data").containsKey("user")) {
                                    continue;
                                }
                                JSONArray array = jsonObject.getJSONObject("data").getJSONArray("user");
                                if (array != null && !array.isEmpty()) {
                                    for (int i = 0; i < array.size(); i++) {
                                        JSONObject object = array.getJSONObject(i);
                                        String id = object.getString("id");
                                        String photo = "http://tx.tianyaui.com/logo/small/" + id;
                                        String name = object.getString("name");
                                        String url_ = "http://www.tianya.cn/" + id;
                                        UserBean result_object = new UserBean();
                                        result_object.setUrl(url_);
                                        result_object.setPublish_user_id(id);
                                        result_object.setPublish_user_photo(photo);
                                        result_object.setPublish_user(name);
                                        result_object.setJson_id(setting.getName() + "_" + id);
                                        fans_details.add(result_object);
                                    }
                                }
                            } catch (Exception e) {
                                log.error("天涯社区获取粉丝数据解析出错:" + follow_details_url, e);
                                e.printStackTrace();
                            }
                        }
                    }
                    data.setFans_details(fans_details);
                    data.setFans_count(fans_count);
                    //如果粉丝数是不是0但是详情为空则不要这条数据
                    if(fans_count > 0 && fans_details.isEmpty()) {
                        return null;
                    }
                    item.put("data", data.toJSONObject());
                    return item;
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error(e.getMessage(), e);
                }
            }
        }
        return null;
    }

    // TODO: 2017/12/27 天涯社区测试
    public static void main(String[] args) throws Exception {
        Task task = new Task("http://www.tianya.cn/8028471", TaskType.USER, WebType.FORLUM, "ddd_8028471", null);
        task.put("publish_user", "8028471");
        task.put("publish_user_photo", "8028471");
        JSONObject object = new TianYaCrawler().crawlItem(task);
        if(CommonUtil.isNotNull(object)) {
            System.out.println(object.toJSONString());
        } else {
            System.exit(1);
        }
    }


    /**
     * 抓回复的线程(这个是个性化的每个网站都不一样)
     */
    public class ReplyCrawler extends AbstractReplyCrawler {

        private ArrayList<Integer> pages;

        @Override
        public List<SpeechBean> call() throws Exception {
            List<SpeechBean> array = new ArrayList<>();
            for (int k = 0; k < pages.size(); k++) {
                try {
                    String[] urlParam = url.split("-");
                    String getUrl = urlParam[0] + "-" + urlParam[1] + "-" + urlParam[2] + "-" + pages.get(k) +
                            urlParam[3].substring(urlParam[3].length() - 6, urlParam[3].length());
                    Document document = null;
                    ResponseEntity entity = downloader.get(getUrl, true);
                    if (entity.getStatus_code() == 200) {
                        document = entity.getDocument();
                        Elements replyEntities = null;
                        Elements bd_elements = document.select("#bd");
                        if (!bd_elements.isEmpty()) {
                            replyEntities = bd_elements.get(0).select("div[class=\"atl-main\"]").select("div[class=\"atl-item\"]");
                            if (!replyEntities.isEmpty()) {
                                for (int i = 0; i < replyEntities.size(); i++) {
                                    Element reply_element = replyEntities.get(i);
                                    Elements replyUserInfo = reply_element.select(".atl-head").select("" + ""
                                            + ".atl-info").select("span");
                                    SpeechBean reply = new SpeechBean();
                                    String replyUser = "";
                                    if (null != replyUserInfo && null != replyUserInfo.get(0).select("a") &&
                                            replyUserInfo.get(0).select("a").size() > 0) {
                                        replyUser = replyUserInfo.get(0).select("a").get(0).text();
                                    }
                                    String replyUserId = replyUserInfo.get(0).select("a").get(0).attr("uid");
                                    String replyUserPhoto = "http://tx.tianyaui.com/logo/" + replyUserId;
                                    replyEntities.get(i).select(".atl-content").get(0).select(".atl-reply").remove();
                                    String replyContent = reply_element.select(".atl-content div[class=\"bbs-content\"]").text();
                                    List<String> reply_img_urls = new ArrayList();
                                    Elements replyImgElemnts = reply_element.select(".atl-content").select("img[original]");
                                    for (int imgIndex = 0; imgIndex < replyImgElemnts.size(); imgIndex++) {
                                        Element imgEle = replyImgElemnts.get(imgIndex);
                                        String img_url = imgEle.attr("original");
                                        reply_img_urls.add(img_url);
                                    }
                                    String replyTimeStr = replyUserInfo.get(1).text();
                                    String replyTimeStr2 = replyTimeStr.substring(3, replyTimeStr.length());
                                    String replyTime = getTime(replyTimeStr2);
                                    String replyUrl = getUrl + "#" + reply_element.id();
                                    String replyId = reply_element.attr("replyid");
                                    reply.setId(replyId);
                                    reply.setPublish_user(replyUser);
                                    reply.setPublish_user_id(replyUserId);
                                    reply.setPublish_user_jsonid(setting.getName() + "_" + replyUserId);
                                    reply.setPublish_user_photo(replyUserPhoto);
                                    if (CommonUtil.isNotNull(replyUserId)) {
                                        if (replyUserId.contains("http")) {
                                            replyUserId = replyUserId.substring(replyUserId.lastIndexOf("/"));
                                        }
                                        if (replyUserId.startsWith("/")) {
                                            replyUserId = replyUserId.substring(1);
                                        }
                                        String userKey = setting.getName() + "_" + replyUserId;
                                        String user_url = "http://www.tianya.cn/" + replyUserId;
                                        Task task = new Task(user_url, TaskType.USER, WebType.FORLUM, userKey);
                                        task.put("publish_user", replyUser);
                                        task.put("publish_user_photo", replyUserPhoto);
                                        TaskManager.add(setting.getUser_task(), task);
                                    }
                                    reply.setContent(replyContent);
                                    reply.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                            replyTime + ":00"));
                                    reply.setUrl(replyUrl);
                                    reply.setParent_id(parentId);
                                    reply.setAncestor_id(ancestorId);
                                    reply.setArea(this.area);
                                    reply.setImg_urls(reply_img_urls);

                                    //获取回复的回复
                                    Elements reply_reply_elements = reply_element.select("div[class=\"item-reply-view\"] ul li");
                                    List<SpeechBean> reply_reply_nodes = new ArrayList<>();
                                    for(Element element : reply_reply_elements) {
                                        SpeechBean reply_reply_bean = new SpeechBean();
                                        String reply_reply_publish_user_id = element.attr("_userid");
                                        reply_reply_bean.setPublish_user_id(reply_reply_publish_user_id);
                                        String reply_reply_publish_time = element.attr("_replytime");
                                        reply_reply_bean.setPublish_time(DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss, reply_reply_publish_time));
                                        Elements p_elements = element.select("p");
                                        if(p_elements.isEmpty()) {
                                            continue;
                                        }

                                        String reply_reply_publish_user = element.attr("_username");
                                        String reply_reply_publish_user_photo = "http://tx.tianyaui.com/logo/" + reply_reply_publish_user_id;
                                        reply_reply_bean.setPublish_user_photo(reply_reply_publish_user_photo);
                                        reply_reply_bean.setPublish_user(reply_reply_publish_user);
                                        Element content_elment = element.select("span[class=\"ir-content\"]").get(0);
                                        String reply_reply_content = content_elment.text();
                                        reply_reply_bean.setContent(reply_reply_content);
                                        reply_reply_bean.setPublish_user_jsonid(setting.getName() + "_" + reply_reply_publish_user_id);
                                        reply_reply_bean.setParent_id(replyId);
                                        reply_reply_bean.setAncestor_id(ancestorId);
                                        reply_reply_bean.setUrl(replyUrl);
                                        reply_reply_nodes.add(reply_reply_bean);
                                    }
                                    reply.setReply_count(reply_reply_nodes.size());
                                    //添加回复的回复
                                    reply.setReply_nodes(reply_reply_nodes);
                                    array.add(reply);
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    log.error("天涯社区帖子回复: " + url + "出错", e);
                }
            }
            return array;
        }

        public ReplyCrawler(ArrayList<Integer> pages, String url, String parentId, String ancestorId, Area area) {
            this.pages = pages;
            this.url = url;
            this.parentId = parentId;
            this.ancestorId = ancestorId;
            this.area = area;
        }
    }


    /**
     * 生成任务时针对那些没有地域的线程(这个是个性化的每个网站都不一样)
     */
    public class TianYaTaskCrawler implements Runnable {

        private Area area = null;

        private String url;

        private Downloader downloader = DownloaderFactory.buildDownloader();

        public TianYaTaskCrawler(String url) {
            this.url = url;
        }

        public TianYaTaskCrawler(Area area, String url) {
            this.area = area;
            this.url = url;
        }


        @Override
        public void run() {
            handle(url);
        }

        /**
         * 当前任务
         *
         * @param url
         */
        private void handle(String url) {
            for (int page = 0; page < 5; page++) {
                //是否抓下一页
                boolean haveLater = false;
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(url, true);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements elements = document.select("table[class=\"tab-bbs-list tab-bbs-list-2\"] tbody tr");
                        for (int i = 0; i < elements.size(); i++) {
                            Element element = elements.get(i);
                            Elements a_list = element.select("a");
                            if (CommonUtil.isNull(a_list)) {
                                continue;
                            }
                            String speech_url = a_list.get(0).attr("href");
                            if (CommonUtil.isNotNull(speech_url)) {
                                if (!speech_url.contains("http")) {
                                    speech_url = "http://bbs.tianya.cn" + speech_url;
                                }
                                String updateTimeString = element.select("td:eq(4)").attr("title");
                                if (CommonUtil.isNotNull(updateTimeString)) {
                                    updateTimeString = updateTimeString + ":00";
                                    Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                            updateTimeString);
                                    //这个key是用来查询该言论是否被爬过的唯一标识
                                    String speechkey = setting.getSite() + "_" + speech_url.substring
                                            (speech_url.lastIndexOf("/") + 1, speech_url.lastIndexOf(".")).trim();
                                    Task task = new Task(speech_url, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                                    TaskManager.add(setting.getSpeech_task(), task);
                                    if (i == elements.size() - 1) {
                                        Date now = new Date();
                                        if (updateTime.getTime() > now.getTime() - Constant.LAST_24_HOUR_TIME_STAMP) {
                                            Elements pageElements = document.select("div[class=\"short-pages-2 " +
                                                    "clearfix\"] div a");
                                            for (Element pageElement : pageElements) {
                                                if ("下一页".equals(pageElement.text())) {
                                                    String laterUrl = pageElement.attr("href");
                                                    if (!laterUrl.contains("http")) {
                                                        laterUrl = index + laterUrl;
                                                    }
                                                    if (!url.equals(laterUrl)) {
                                                        haveLater = true;
                                                        url = laterUrl;
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    //不存在下一页退出循环
                    if (!haveLater) {
                        break;
                    }
                } catch (Exception e) {
                    log.error(setting.getName() + "生成任务出错: ", e);
                }
            }
        }


        /**
         * 历史任务
         *
         * @param url
         */
        private void history(String url) {
            for (int page = 0; page < 1000; page++) {
                try {
                    ResponseEntity entity = null;
                    try {
                        entity = downloader.get(url, true);
                    } catch (Exception e) {
                        log.error(e.getMessage(), e);
                    }
                    if (entity != null && entity.getStatus_code() == 200) {
                        Document document = entity.getDocument();
                        Elements elements = document.select("table[class=\"tab-bbs-list tab-bbs-list-2\"] tbody tr");
                        for (int i = 0; i < elements.size(); i++) {
                            Element element = elements.get(i);
                            Elements a_list = element.select("a");
                            if (CommonUtil.isNull(a_list)) {
                                continue;
                            }
                            String speech_url = a_list.get(0).attr("href");
                            if (CommonUtil.isNotNull(speech_url)) {
                                if (!speech_url.contains("http")) {
                                    speech_url = "http://bbs.tianya.cn" + speech_url;
                                }
                                String update_time_string = element.select("td:eq(4)").attr("title");
                                if (CommonUtil.isNotNull(update_time_string)) {
                                    update_time_string = update_time_string + ":00";
                                    Date updateTime = DateUtil.str_to_date(DateUtil.yyyy_MM_dd_HH_mm_ss,
                                            update_time_string);
                                    //这个key是用来查询该言论是否被爬过的唯一标识
                                    String speechkey = setting.getSite() + "_" + speech_url.substring
                                            (speech_url.lastIndexOf("/") + 1, speech_url.lastIndexOf(".")).trim();
                                    Task task = new Task(speech_url, TaskType.SPEECH, WebType.FORLUM, speechkey, area);
                                    TaskManager.add(setting.getSpeech_task(), task);
                                    Elements pageElements = document.select("div[class=\"short-pages-2 clearfix\"] "
                                            + "div a");
                                    if (pageElements.isEmpty()) {
                                        break;
                                    }
                                    for (Element pageElement : pageElements) {
                                        if ("下一页".equals(pageElement.text())) {
                                            String laterUrl = pageElement.attr("href");
                                            if (!laterUrl.contains("http")) {
                                                laterUrl = index + laterUrl;
                                            }
                                            if (url.equals(laterUrl)) {
                                                break;
                                            }
                                            url = laterUrl;
                                        }
                                    }
                                }
                            }
                        }
                    }
                } catch (Exception e) {
                    log.error("天涯社区生成任务出错: ", e);
                }
            }
        }

    }

}
